{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![seq2seq_attention_intent_slot模型](img/img2.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n对话中的意图识别和槽填充联合模型：\\n这里实现了《Attention-Based Recurrent Neural Network Models for Joint Intent Detection and Slot Filling》上图中(c)模型\\n此模型利用seq2seq-attention实现：\\n1.意图识别是利用encoder中的最后一个time step中的双向隐层 + encoder的attention，最后接一个fc层进行分类\\n2.槽填充利用序列标注，基于attention的常方法，最后也是一个fc层分类\\n3.总的loss = 意图识别loss + 槽填充loss\\n'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "对话中的意图识别和槽填充联合模型：\n",
    "这里实现了《Attention-Based Recurrent Neural Network Models for Joint Intent Detection and Slot Filling》上图中(c)模型\n",
    "此模型利用seq2seq-attention实现：\n",
    "1.意图识别是利用encoder中的最后一个time step中的双向隐层 + encoder的attention，最后接一个fc层进行分类\n",
    "2.槽填充利用序列标注，基于attention的常方法，最后也是一个fc层分类\n",
    "3.总的loss = 意图识别loss + 槽填充loss\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from torchtext import data, datasets\n",
    "import pandas as pd\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_dir = os.path.abspath(os.path.join(os.getcwd(), \"..\"))\n",
    "atis_data = os.path.join(base_dir, 'atis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "build train and val dataset\n",
    "'''\n",
    "    \n",
    "tokenize = lambda s:s.split()\n",
    "\n",
    "SOURCE = data.Field(sequential=True, tokenize=tokenize,\n",
    "                    lower=True, use_vocab=True,\n",
    "                    init_token='<sos>', eos_token='<eos>',\n",
    "                    pad_token='<pad>', unk_token='<unk>',\n",
    "                    batch_first=True, fix_length=50,\n",
    "                    include_lengths=True) #include_lengths=True为方便之后使用torch的pack_padded_sequence\n",
    "\n",
    "TARGET = data.Field(sequential=True, tokenize=tokenize,\n",
    "                    lower=True, use_vocab=True,\n",
    "                    init_token='<sos>', eos_token='<eos>',\n",
    "                    pad_token='<pad>', unk_token='<unk>',\n",
    "                    batch_first=True, fix_length=50,\n",
    "                    include_lengths=True) #include_lengths=True为方便之后使用torch的pack_padded_sequence\n",
    "LABEL = data.Field(\n",
    "                sequential=False,\n",
    "                use_vocab=True)\n",
    "\n",
    "train, val = data.TabularDataset.splits(\n",
    "                                        path=atis_data,\n",
    "                                        skip_header=True,\n",
    "                                        train='atis.train.csv',\n",
    "                                        validation='atis.test.csv',\n",
    "                                        format='csv',\n",
    "                                        fields=[('index', None), ('intent', LABEL), ('source', SOURCE), ('target', TARGET)])\n",
    "\n",
    "SOURCE.build_vocab(train, val)\n",
    "TARGET.build_vocab(train, val)\n",
    "LABEL.build_vocab(train, val)\n",
    "\n",
    "train_iter, val_iter = data.Iterator.splits(\n",
    "                                            (train, val),\n",
    "                                            batch_sizes=(64, len(val)), # 训练集设置为64,验证集整个集合用于测试\n",
    "                                            shuffle=True,\n",
    "                                            sort_within_batch=True, #为true则一个batch内的数据会按sort_key规则降序排序\n",
    "                                            sort_key=lambda x: len(x.source)) #这里按src的长度降序排序，主要是为后面pack,pad操作)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save source words\n",
    "source_words_path = os.path.join(os.getcwd(), 'source_words.pkl')\n",
    "with open(source_words_path, 'wb') as f_source_words:\n",
    "    pickle.dump(SOURCE.vocab, f_source_words)\n",
    "\n",
    "# save target words\n",
    "target_words_path = os.path.join(os.getcwd(), 'target_words.pkl')\n",
    "with open(target_words_path, 'wb') as f_target_words:\n",
    "    pickle.dump(TARGET.vocab, f_target_words)\n",
    "    \n",
    "# save label words\n",
    "label_words_path = os.path.join(os.getcwd(), 'label_words.pkl')\n",
    "with open(label_words_path, 'wb') as f_label_words:\n",
    "    pickle.dump(LABEL.vocab, f_label_words)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "import math\n",
    "from apex import amp\n",
    "import time\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build model\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "# 构建编码器\n",
    "class Encoder(nn.Module):\n",
    "    def __init__(self, input_dim, emb_dim, hidden_dim, n_layers, dropout, pad_index):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.pad_index = pad_index\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.n_layers = n_layers\n",
    "        \n",
    "        self.embedding = nn.Embedding(input_dim, emb_dim, padding_idx=pad_index)\n",
    "        self.gru = nn.GRU(emb_dim, hidden_dim, n_layers, dropout=dropout, bidirectional=True, batch_first=True) #使用双向\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        self.fc = nn.Linear(hidden_dim * 2, hidden_dim)\n",
    "    def forward(self, src, src_len):\n",
    "        # 初始化\n",
    "        # h0 = torch.zeros(self.n_layers, src.size(1), self.hidden_dim).to(device)\n",
    "        # c0 = torch.zeros(self.n_layers, src.size(1), self.hidden_dim).to(device)\n",
    "        # nn.init.kaiming_normal_(h0)\n",
    "        # nn.init.kaiming_normal_(c0)\n",
    "        # src=[batch_size, seq_len]\n",
    "        embedded = self.dropout(self.embedding(src))\n",
    "        # embedd=[batch_size,seq_len,embdim]\n",
    "        packed = torch.nn.utils.rnn.pack_padded_sequence(embedded, src_len, batch_first=True, enforce_sorted=True) #这里enfore_sotred=True要求数据根据词数排序\n",
    "        output, hidden = self.gru(packed)\n",
    "        # output=[batch_size, seq_len, hidden_size*2]\n",
    "        # hidden=[n_layers*2, batch_size, hidden_size]\n",
    "        \n",
    "        output, _ = torch.nn.utils.rnn.pad_packed_sequence(output, batch_first=True, padding_value=self.pad_index, total_length=len(src[0])) #这个会返回output以及压缩后的legnths\n",
    "        \n",
    "        '''\n",
    "        hidden[-2,:,:]是gru最后一步的forward\n",
    "        hidden[-1,:,:]是gru最后一步的backward\n",
    "        利用最后前向和后向的hidden的隐状态作为decoder的初始状态\n",
    "        hidden:[batch_size, hidden_dim]\n",
    "        '''\n",
    "        hidden = torch.tanh(self.fc(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim=1)))\n",
    "        return output, hidden\n",
    "\n",
    "# 构建attention权重计算方式\n",
    "class Attention(nn.Module):\n",
    "    def __init__(self, hidden_dim):\n",
    "        super(Attention, self).__init__()\n",
    "        self.attn = nn.Linear((hidden_dim * 2) + hidden_dim, hidden_dim)\n",
    "        self.v = nn.Linear(hidden_dim, 1, bias=False)\n",
    "\n",
    "    def concat_score(self, hidden, encoder_output):\n",
    "        seq_len = encoder_output.shape[1]\n",
    "        hidden = hidden.unsqueeze(1).repeat(1, seq_len, 1) # [batch_size, seq_len, hidden_size]\n",
    "        energy = torch.tanh(self.attn(torch.cat((hidden, encoder_output),dim=2))) # [batch_size, seq_len, hidden_dim]\n",
    "        attention = self.v(energy).squeeze(2) #[batch_size, seq_len]\n",
    "        return attention #[batch_size, seq_len]\n",
    "\n",
    "    def forward(self, hidden, encoder_output):\n",
    "        # hidden = [batch_size, hidden_size]\n",
    "        # #encoder_output=[batch_size, seq_len, hidden_dim*2]\n",
    "        \n",
    "        attn_energies = self.concat_score(hidden, encoder_output)\n",
    "\n",
    "        return F.softmax(attn_energies, dim=1).unsqueeze(1) #softmax归一化，[batch_size, 1, seq_len]\n",
    "\n",
    "# 构建解码器\n",
    "class Decoder(nn.Module):\n",
    "    def __init__(self, output_dim, emb_dim, hidden_dim, n_layers, dropout):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.output_dim = output_dim\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.n_layers = n_layers\n",
    "\n",
    "        self.embedding = nn.Embedding(output_dim, emb_dim)\n",
    "        self.gru = nn.GRU((hidden_dim * 2) + emb_dim + (hidden_dim * 2), hidden_dim, n_layers, dropout=dropout, batch_first=True)\n",
    "      \n",
    "        self.attention = Attention(hidden_dim)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "    def forward(self, input, hidden, encoder_output, aligned):\n",
    "        input = input.unsqueeze(1)\n",
    "        # input=[batch_size, 1]\n",
    "        # hidden=[batch_size, hidden_size] 初始化为encoder的最后一层 [batch_size, hidden_size]\n",
    "        # encoder_output=[batch_size, seq_len, hidden_dim*2]\n",
    "        # aligned=[batch_size, 1, hidden_dim*2]\n",
    "        \n",
    "        # embedded=[batch_sze, 1, emb_dim]\n",
    "        embedded = self.dropout(self.embedding(input))\n",
    "\n",
    "        # 利用利用上一步的hidden与encoder_output，计算attention权重\n",
    "        # attention_weights=[batch_size, 1, seq_len]\n",
    "        attention_weights = self.attention(hidden, encoder_output)\n",
    "\n",
    "        '''\n",
    "        以下是计算上下文：利用attention权重与encoder_output计算attention上下文向量\n",
    "        注意力权重分布用于产生编码器隐藏状态的加权和，加权平均的过程。得到的向量称为上下文向量\n",
    "        '''\n",
    "        context = attention_weights.bmm(encoder_output) # [batch_size, 1, seq_len]*[batch_size,seq_len,hidden_dim*2]=[batch_size, 1, hidden_dim*2]\n",
    "        \n",
    "        #拼接注意力上下文和embedding向量，以及encoder输出的hidden对齐向量作为gru输入\n",
    "        # [batch_size, 1, hidden_dim*2+emb_dim+hidden_dim*2]\n",
    "        gru_input = torch.cat([context, embedded, aligned], 2)\n",
    "        \n",
    "        # 将注意力向量，本次embedding以及上次的hidden输入到ｇｒｕ中\n",
    "        # decoder_output=[batch_size, seq_len, hidden_size]\n",
    "        # hidden=[n_layers, batch_size, hidden_size]\n",
    "        # decoder中的ｇｒｕ是单向，序列长度为１，层为１，\n",
    "        # 所以decoder_output=[batch_size, １, hidden_size]，hidden=[１, batch_size, hidden_size]\n",
    "        decoder_output, hidden = self.gru(gru_input, hidden.unsqueeze(0))\n",
    "        \n",
    "\n",
    "        decoder_output_context = torch.cat([decoder_output, context], 2) # 连接context与decoder_output的hidden_dim =[batch_size, 1, 2 * hidden_dim + hidden_dim]\n",
    "      \n",
    "        # prediction=[batch_size, output_dim]，词汇表中所有词的概率分布，这里可以使用softmax进行归一化\n",
    "        return decoder_output_context.squeeze(1), hidden.squeeze(0), attention_weights.squeeze(1), context.squeeze(1)\n",
    "\n",
    "# 利用Encoder与Decoder构建seq2seq模型\n",
    "class Seq2Seq(nn.Module):\n",
    "    '''\n",
    "    接收source句子\n",
    "    利用编码器encoder生成上下文向量\n",
    "    利用解码器decoder生成预测target句子\n",
    "\n",
    "    每次迭代中：\n",
    "    传入input以及先前的hidden与cell状态给解码器decoder\n",
    "    从解码器decoder中接收一个prediction以及下一个hidden与下一个cell状态\n",
    "    保存这个prediction作为预测句子中的一部分\n",
    "    决定是否使用\"teacher force\":\n",
    "        如果使用：解码器的下一次input是真实的token\n",
    "        如果不使用：解码器的下一次input是预测prediction（使用output tensor的argmax）的token\n",
    "    '''\n",
    "\n",
    "    def __init__(self, predict_flag, encoder, decoder, intent_size, output_dim):\n",
    "        super(Seq2Seq, self).__init__()\n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "        self.predict_flag = predict_flag\n",
    "        # 意图分类\n",
    "        self.intent_out = nn.Linear((encoder.hidden_dim * 2) + encoder.hidden_dim, intent_size)\n",
    "          # 槽填充slot filling\n",
    "        self.slot_out = nn.Linear(hidden_dim * 2 + hidden_dim, output_dim)\n",
    "        assert encoder.hidden_dim == decoder.hidden_dim, 'encoder与decoder的隐藏状态维度必须相等！'\n",
    "        assert encoder.n_layers == decoder.n_layers, 'encoder与decoder的层数必须相等！'\n",
    "        \n",
    "    def forward(self, src, src_lens, trg, teacher_forcing_ration=1.0):\n",
    "        '''\n",
    "        src=[batch_size, seq_len]\n",
    "        src_len=[batch_size]\n",
    "        trg=[batch_size, trg_len]\n",
    "        \n",
    "        '''\n",
    "        # 预测，一次输入一句话\n",
    "        if self.predict_flag:\n",
    "            assert len(src) == 1, '预测时一次输入一句话'\n",
    "            src_len = len(src[0])\n",
    "            output_tokens = []\n",
    "            encoder_output, encoder_hidden = self.encoder(src, src_lens)\n",
    "            aligns = encoder_output.transpose(0,1) #对齐向量\n",
    "            \n",
    "            hidden = encoder_hidden\n",
    "            input = torch.tensor(2).unsqueeze(0)  # 预测阶段解码器输入第一个token-> <sos>\n",
    "            for s in range(src_len):\n",
    "                aligned = aligns[s].unsqueeze(1)# [batcg_size, 1, hidden_size*2]\n",
    "                if s == 0:\n",
    "                    # context = [batch_size, hidden_dim*2]\n",
    "                    decoder_output_context, hidden, _, context = self.decoder(input, hidden, encoder_output, aligned)\n",
    "                else:\n",
    "                    decoder_output_context, hidden, _, _ = self.decoder(input, hidden, encoder_output, aligned)\n",
    "                 # 槽,[batch_size, output_dim]\n",
    "                output = self.slot_out(decoder_output_context)\n",
    "                input = output.argmax(1)\n",
    "                output_token = input.squeeze().detach().item()\n",
    "               \n",
    "                output_tokens.append(output_token)\n",
    "            concated = torch.cat((encoder_hidden, context), 1)\n",
    "            intent_outputs = self.intent_out(concated)\n",
    "            intent_outputs = intent_outputs.squeeze()\n",
    "            intent_outputs = intent_outputs.argmax()\n",
    "            return output_tokens, intent_outputs\n",
    "\n",
    "        # 训练\n",
    "        else:\n",
    "            '''\n",
    "            src=[batch_size, seq_len]\n",
    "            trg=[batch_size, trg_len]\n",
    "            teacher_forcing_ration是使用teacher forcing的概率,例如teacher_forcing_ration=0.8，则输入的时间步有80%的真实值。\n",
    "            '''\n",
    "            batch_size = trg.shape[0]\n",
    "            trg_len = trg.shape[1]\n",
    "            trg_vocab_size = self.decoder.output_dim\n",
    "            # 存储decoder outputs\n",
    "            slot_outputs = torch.zeros(batch_size, trg_len, trg_vocab_size).to(device)\n",
    "            # encoder的最后一层hidden state(前向＋后向)作为decoder的初始隐状态,[batch_size, seq_len, hidden_size*2]\n",
    "            # hidden=[batch_size, hidden_size]\n",
    "            encoder_output, encoder_hidden = self.encoder(\n",
    "                src, src_lens)  \n",
    "            hidden = encoder_hidden\n",
    "            \n",
    "            # 输入到decoder的第一个是<sos>\n",
    "            input = trg[:, 0] # [batch_size]\n",
    "            \n",
    "            aligns = encoder_output.transpose(0,1) #对齐向量\n",
    "            \n",
    "            for t in range(1, trg_len):\n",
    "                '''\n",
    "                解码器输入的初始hidden为encoder的最后一步的hidden\n",
    "                接收输出即predictions和新的hidden状态\n",
    "                '''\n",
    "                aligned = aligns[t].unsqueeze(1)# [batcg_size, 1, hidden_size*2]\n",
    "                \n",
    "                if t == 1:\n",
    "                    # context = [batch_size, hidden_dim*2]\n",
    "                    decoder_output_context, hidden, _, context = self.decoder(input, hidden, encoder_output, aligned)\n",
    "                else:\n",
    "                    decoder_output_context, hidden, _, _ = self.decoder(input, hidden, encoder_output, aligned)\n",
    "                # 槽,[batch_size, output_dim]\n",
    "                output = self.slot_out(decoder_output_context)\n",
    "                # 存入decoder的预测值\n",
    "                slot_outputs[:, t, :] = output\n",
    "                # 是否使用teacher forcing\n",
    "                teacher_force = random.random() < teacher_forcing_ration\n",
    "                # 获取预测的最大概率的token\n",
    "                predict_max = output.argmax(1)\n",
    "                '''\n",
    "                如果是teacher forcing则下一步使用真实token作为解码的输入\n",
    "                否则使用decoder的预测值作为下一步的解码输入\n",
    "                '''\n",
    "                input = trg[:, t] if teacher_force else predict_max\n",
    "            # concated = [batch_size, hidden_dim * 2 + hidden_dim]\n",
    "            concated = torch.cat((encoder_hidden, context), 1)\n",
    "            intent_outputs = self.intent_out(concated)\n",
    "            # slot_outputs=[batch_size, trg_len, trg_vocab_size], intetn_outputs=[batch_size, intent_size]\n",
    "            return slot_outputs, intent_outputs\n",
    "\n",
    "\n",
    "# 构建模型，优化函数，损失函数，学习率衰减函数\n",
    "def build_model(source, target, label, encoder_embedding_dim, decoder_embedding_dim, hidden_dim, n_layers, encoder_dropout,\n",
    "                decoder_dropout, lr, gamma, weight_decay):\n",
    "    '''\n",
    "    训练seq2seq model\n",
    "    input与output的维度是字典的大小。\n",
    "    encoder与decoder的embedding与dropout可以不同\n",
    "    网络的层数与hiden/cell状态的size必须相同\n",
    "    '''\n",
    "    input_dim = len(source.vocab) # source 词典大小（即词数量）\n",
    "    output_dim = len(target.vocab) # target 词典大小（即实体类型数量）\n",
    "    label_dim = len(label.vocab) # label 词典大小（即意图类别数量）\n",
    "    \n",
    "    encoder = Encoder(input_dim, encoder_embedding_dim, hidden_dim, n_layers, encoder_dropout, source.vocab.stoi[source.pad_token])\n",
    "    decoder = Decoder(output_dim, decoder_embedding_dim, hidden_dim, n_layers, decoder_dropout)\n",
    "\n",
    "    model = Seq2Seq(False, encoder, decoder, label_dim, output_dim).to(device)\n",
    "\n",
    "    model.apply(init_weights)\n",
    "\n",
    "    # 定义优化函数\n",
    "    # optimizer = optim.Adam(model.parameters(), lr=lr) #, weight_decay=weight_decay)\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=lr) #, momentum=0.9, nesterov=True)\n",
    "    # 定义lr衰减\n",
    "    #scheduler = lr_scheduler.ExponentialLR(optimizer, gamma=gamma)\n",
    "    \n",
    "    '''\n",
    "        当网络的评价指标不在提升的时候，可以通过降低网络的学习率来提高网络性能:\n",
    "        optimer指的是网络的优化器\n",
    "        mode (str) ，可选择‘min’或者‘max’，min表示当监控量停止下降的时候，学习率将减小，max表示当监控量停止上升的时候，学习率将减小。默认值为‘min’\n",
    "        factor 学习率每次降低多少，new_lr = old_lr * factor\n",
    "        patience=10，容忍网路的性能不提升的次数，高于这个次数就降低学习率\n",
    "        verbose（bool） - 如果为True，则为每次更新向stdout输出一条消息。 默认值：False\n",
    "        threshold（float） - 测量新最佳值的阈值，仅关注重大变化。 默认值：1e-4\n",
    "        cooldown(int)： 冷却时间“，当调整学习率之后，让学习率调整策略冷静一下，让模型再训练一段时间，再重启监测模式。\n",
    "        min_lr(float or list):学习率下限，可为 float，或者 list，当有多个参数组时，可用 list 进行设置。\n",
    "        eps(float):学习率衰减的最小值，当学习率变化小于 eps 时，则不调整学习率。\n",
    "    '''\n",
    "    scheduler = lr_scheduler.ReduceLROnPlateau(optimizer=optimizer, mode='min', factor=0.1, patience=2, verbose=False)\n",
    "    # 这里忽略<pad>的损失。\n",
    "    target_pad_index = target.vocab.stoi[source.pad_token]\n",
    "    # 定义损失函数(实体识别)\n",
    "    loss_slot = nn.CrossEntropyLoss(ignore_index=target_pad_index)\n",
    "    # 定义损失函数(意图识别)\n",
    "    loss_intent = nn.CrossEntropyLoss()\n",
    "    \n",
    "    return model, optimizer, scheduler, loss_slot, loss_intent\n",
    "\n",
    "\n",
    "# 训练\n",
    "def train(model, iterator, optimizer, loss_slot, loss_intent, clip):\n",
    "    '''\n",
    "    开始训练：\n",
    "        1.得到source与target句子\n",
    "        2.上一批batch的计算梯度归0\n",
    "        3.给模型喂source与target，并得到输出output\n",
    "        4.由于损失函数只适用于带有1维target和2维的input，我们需要用view进行flatten(在计算损失时，从output与target中忽略了第一列<sos>)\n",
    "        5.反向传播计算梯度loss.backward()\n",
    "        6.梯度裁剪，防止梯度爆炸\n",
    "        7.更新模型参数\n",
    "        8.损失值求和(返回所有batch的损失的均值)\n",
    "    '''\n",
    "    model.train()\n",
    "    epoch_loss = 0\n",
    "\n",
    "    for i, batch in enumerate(iterator):\n",
    "        src, src_lens = batch.source  # src=[batch_size, seq_len]，这里batch.src返回src和src的长度，因为在使用torchtext.Field时设置include_lengths=True\n",
    "        trg, _ = batch.target  # trg=[batch_size, seq_len]\n",
    "        label = batch.intent # [batch_size]\n",
    "        src = src.to(device)\n",
    "        trg = trg.to(device)\n",
    "        label = label.to(device)\n",
    "        \n",
    "        #slot_outputs=[batch_size, trg_len, trg_vocab_size], intetn_outputs=[batch_size, intent_size]\n",
    "        slot_outputs, intent_outputs = model(src, src_lens, trg, teacher_forcing_ration=1.0)\n",
    "        \n",
    "        # 以下在计算损失时，忽略了每个tensor的第一个元素及<sos>\n",
    "        output_dim = slot_outputs.shape[-1]\n",
    "        slot_outputs = slot_outputs[:, 1:, :].reshape(-1, output_dim)  # output=[batch_size * (seq_len - 1), output_dim]\n",
    "        trg = trg[:, 1:].reshape(-1)  # trg=[batch_size * (seq_len - 1)]\n",
    "        loss1 = loss_slot(slot_outputs, trg)\n",
    "        loss2 = loss_intent(intent_outputs, label)\n",
    "        loss = loss1 + loss2\n",
    "        \n",
    "        with amp.scale_loss(loss, optimizer) as scaled_loss:\n",
    "            scaled_loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "        epoch_loss += float(loss.item())\n",
    "        # print('epoch_loss:{}'.format(float(loss.item())))\n",
    "    return epoch_loss / len(iterator)\n",
    "\n",
    "'''\n",
    "评估\n",
    "'''\n",
    "def evaluate(model, iterator, loss_slot, loss_intent):\n",
    "    model.eval()  # 评估模型，切断dropout与batchnorm\n",
    "    epoch_loss = 0\n",
    "    with torch.no_grad():  # 不更新梯度\n",
    "        for i, batch in enumerate(iterator):\n",
    "            src, src_len = batch.source  # src=[batch_size, seq_len]\n",
    "            trg, _ = batch.target  # trg=[batch_size, seq_len]\n",
    "            label = batch.intent\n",
    "            src = src.to(device)\n",
    "            trg = trg.to(device)\n",
    "            label = label.to(device)\n",
    "            # output=[batch_size, seq_len, output_dim]\n",
    "            slot_outputs, intent_outputs = model(src, src_len, trg, teacher_forcing_ration=0)  # 评估的时候不使用teacher force，使用预测作为每一步的输入\n",
    "\n",
    "            output_dim = slot_outputs.shape[-1]\n",
    "            slot_outputs = slot_outputs[:, 1:, :].reshape(-1, output_dim)  # output=[batch_size * (seq_len - 1), output_dim]\n",
    "            trg = trg[:, 1:].reshape(-1)  # trg=[batch_size * (seq_len - 1)]\n",
    "            loss1 = loss_slot(slot_outputs, trg)\n",
    "            loss2 = loss_intent(intent_outputs, label)\n",
    "            loss = loss1 + loss2\n",
    "            epoch_loss += float(loss.item())\n",
    "    return epoch_loss / len(iterator)\n",
    "\n",
    "\n",
    "def train_model(model, train_iterator, val_iterator, optimizer, scheduler, loss_slot, loss_intent, n_epochs, clip, model_path, writer):\n",
    "    '''\n",
    "    开始训练我们的模型：\n",
    "    1.每一次epoch，都会检查模型是否达到的最佳的validation loss，如果达到了，就更新\n",
    "    最好的validation loss以及保存模型参数\n",
    "    2.打印每个epoch的loss以及困惑度。\n",
    "    '''\n",
    "    best_valid_loss = float('inf')\n",
    "    for epoch in range(n_epochs):\n",
    "        start_time = time.time()\n",
    "        train_loss = train(model, train_iterator, optimizer, loss_slot, loss_intent, clip)\n",
    "        writer.add_scalar('loss',train_loss,global_step=epoch+1)\n",
    "        \n",
    "        valid_loss = evaluate(model, val_iterator, loss_slot, loss_intent)\n",
    "        end_time = time.time()\n",
    "        epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "\n",
    "        if valid_loss < best_valid_loss:\n",
    "            best_valid_loss = valid_loss\n",
    "            torch.save(model.state_dict(), model_path)\n",
    "        scheduler.step(valid_loss)\n",
    "        print('epoch:{},time-mins:{},time-secs:{}'.format(epoch + 1, epoch_mins, epoch_secs))\n",
    "        print('train loss:{},train perplexity:{}'.format(train_loss, math.exp(train_loss)))\n",
    "        print('val loss:{}, val perplexity:{}'.format(valid_loss, math.exp(valid_loss)))\n",
    "    writer.flush()\n",
    "    writer.close()\n",
    "\n",
    "    #每个epoch所花时间\n",
    "def epoch_time(start_time, end_time):\n",
    "    run_tim = end_time - start_time\n",
    "    run_mins = int(run_tim / 60)\n",
    "    run_secs = int(run_tim-(run_mins * 60))\n",
    "    return run_mins,run_secs\n",
    "\n",
    "#对所有模块和子模块进行权重初始化\n",
    "def init_weights(model):\n",
    "    for name,param in model.named_parameters():\n",
    "        nn.init.uniform_(param.data, -0.08, 0.08)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\torch\\nn\\modules\\rnn.py:50: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1\n",
      "  \"num_layers={}\".format(dropout, num_layers))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected optimization level O1:  Insert automatic casts around Pytorch functions and Tensor methods.\n",
      "\n",
      "Defaults for this optimization level are:\n",
      "enabled                : True\n",
      "opt_level              : O1\n",
      "cast_model_type        : None\n",
      "patch_torch_functions  : True\n",
      "keep_batchnorm_fp32    : None\n",
      "master_weights         : None\n",
      "loss_scale             : dynamic\n",
      "Processing user overrides (additional kwargs that are not None)...\n",
      "After processing overrides, optimization options are:\n",
      "enabled                : True\n",
      "opt_level              : O1\n",
      "cast_model_type        : None\n",
      "patch_torch_functions  : True\n",
      "keep_batchnorm_fp32    : None\n",
      "master_weights         : None\n",
      "loss_scale             : dynamic\n",
      "Warning:  multi_tensor_applier fused unscale kernel is unavailable, possibly because apex was installed without --cuda_ext --cpp_ext. Using Python fallback.  Original ImportError was: ModuleNotFoundError(\"No module named 'amp_C'\",)\n",
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 32768.0\n",
      "epoch:1,time-mins:0,time-secs:47\n",
      "train loss:4.380343155983167,train perplexity:79.86543500517435\n",
      "val loss:3.3329546451568604, val perplexity:28.021011659291432\n",
      "epoch:2,time-mins:0,time-secs:47\n",
      "train loss:2.899424436764839,train perplexity:18.163688009262657\n",
      "val loss:3.18021821975708, val perplexity:24.05200160137639\n",
      "epoch:3,time-mins:0,time-secs:47\n",
      "train loss:2.7171691014216495,train perplexity:15.13740906238432\n",
      "val loss:3.05536150932312, val perplexity:21.22885857018245\n",
      "epoch:4,time-mins:0,time-secs:47\n",
      "train loss:2.6233993279628263,train perplexity:13.782495265451914\n",
      "val loss:2.975536346435547, val perplexity:19.600132892352395\n",
      "epoch:5,time-mins:0,time-secs:47\n",
      "train loss:2.5113096130199923,train perplexity:12.32105531374386\n",
      "val loss:2.7806482315063477, val perplexity:16.12947319262866\n",
      "epoch:6,time-mins:0,time-secs:47\n",
      "train loss:2.338704833617577,train perplexity:10.367799837403576\n",
      "val loss:2.5861663818359375, val perplexity:13.27876817495428\n",
      "epoch:7,time-mins:0,time-secs:47\n",
      "train loss:2.176082120491908,train perplexity:8.811715302553653\n",
      "val loss:2.5210347175598145, val perplexity:12.441463406363576\n",
      "epoch:8,time-mins:0,time-secs:47\n",
      "train loss:2.022266242748652,train perplexity:7.555427980387728\n",
      "val loss:2.2865631580352783, val perplexity:9.841057334998489\n",
      "epoch:9,time-mins:0,time-secs:47\n",
      "train loss:1.8627870067571983,train perplexity:6.441664740298277\n",
      "val loss:2.1784090995788574, val perplexity:8.832243855276062\n",
      "epoch:10,time-mins:0,time-secs:47\n",
      "train loss:1.7230180074007084,train perplexity:5.601408071087707\n",
      "val loss:2.0274176597595215, val perplexity:7.594449562538083\n",
      "epoch:11,time-mins:0,time-secs:48\n",
      "train loss:1.6202019177950346,train perplexity:5.054110728434889\n",
      "val loss:1.9293537139892578, val perplexity:6.885059086006016\n",
      "epoch:12,time-mins:0,time-secs:47\n",
      "train loss:1.517061554468595,train perplexity:4.558809680481447\n",
      "val loss:1.8575327396392822, val perplexity:6.407907276098485\n",
      "epoch:13,time-mins:0,time-secs:47\n",
      "train loss:1.4442160282379541,train perplexity:4.238527953727138\n",
      "val loss:1.7986037731170654, val perplexity:6.041206677988556\n",
      "epoch:14,time-mins:0,time-secs:48\n",
      "train loss:1.3496230007746282,train perplexity:3.8559715583504173\n",
      "val loss:1.6869826316833496, val perplexity:5.403152781826219\n",
      "epoch:15,time-mins:0,time-secs:47\n",
      "train loss:1.256624242434135,train perplexity:3.5135405839140406\n",
      "val loss:1.595689058303833, val perplexity:4.931726148447705\n",
      "epoch:16,time-mins:0,time-secs:47\n",
      "train loss:1.1894612319958515,train perplexity:3.2853107101879897\n",
      "val loss:1.5112000703811646, val perplexity:4.532166450796832\n",
      "epoch:17,time-mins:0,time-secs:47\n",
      "train loss:1.1141395331957402,train perplexity:3.0469452556283287\n",
      "val loss:1.4359409809112549, val perplexity:4.203598654436054\n",
      "epoch:18,time-mins:0,time-secs:48\n",
      "train loss:1.0790607264408698,train perplexity:2.941914989965918\n",
      "val loss:1.4073525667190552, val perplexity:4.085125976835411\n",
      "epoch:19,time-mins:0,time-secs:47\n",
      "train loss:1.0202060655141487,train perplexity:2.7737662826522924\n",
      "val loss:1.3761541843414307, val perplexity:3.9596442459180414\n",
      "epoch:20,time-mins:0,time-secs:47\n",
      "train loss:0.9835531826202686,train perplexity:2.6739403810015796\n",
      "val loss:1.3231048583984375, val perplexity:3.755062232815381\n",
      "epoch:21,time-mins:0,time-secs:48\n",
      "train loss:0.9498702937211746,train perplexity:2.585374298287499\n",
      "val loss:1.2544364929199219, val perplexity:3.505862239418173\n",
      "epoch:22,time-mins:0,time-secs:48\n",
      "train loss:0.9107061945475065,train perplexity:2.4860775680371536\n",
      "val loss:1.2352162599563599, val perplexity:3.4391221847034346\n",
      "epoch:23,time-mins:0,time-secs:47\n",
      "train loss:0.8752587682161576,train perplexity:2.3994961269701625\n",
      "val loss:1.1902775764465332, val perplexity:3.2879937503484875\n",
      "epoch:24,time-mins:0,time-secs:47\n",
      "train loss:0.8414802108055506,train perplexity:2.319798227070125\n",
      "val loss:1.1623010635375977, val perplexity:3.1972819669249155\n",
      "epoch:25,time-mins:0,time-secs:47\n",
      "train loss:0.8080090949168572,train perplexity:2.2434370673041952\n",
      "val loss:1.1139987707138062, val perplexity:3.046516390236616\n",
      "epoch:26,time-mins:0,time-secs:47\n",
      "train loss:0.7751921002681439,train perplexity:2.171009138565822\n",
      "val loss:1.1062488555908203, val perplexity:3.022997399596867\n",
      "epoch:27,time-mins:0,time-secs:47\n",
      "train loss:0.7492574713169,train perplexity:2.1154286668365536\n",
      "val loss:1.0564000606536865, val perplexity:2.8759989094062997\n",
      "epoch:28,time-mins:0,time-secs:47\n",
      "train loss:0.7192385020928506,train perplexity:2.0528693595638368\n",
      "val loss:1.0550951957702637, val perplexity:2.8722485668019693\n",
      "epoch:29,time-mins:0,time-secs:47\n",
      "train loss:0.6970677016637264,train perplexity:2.0078564327997617\n",
      "val loss:1.0699535608291626, val perplexity:2.9152441153139415\n",
      "epoch:30,time-mins:0,time-secs:47\n",
      "train loss:0.6702110977509083,train perplexity:1.954649899284747\n",
      "val loss:0.9773122668266296, val perplexity:2.657304509778153\n",
      "epoch:31,time-mins:0,time-secs:47\n",
      "train loss:0.6454820373119452,train perplexity:1.9069060076092446\n",
      "val loss:0.99099200963974, val perplexity:2.69390552752583\n",
      "epoch:32,time-mins:0,time-secs:47\n",
      "train loss:0.6272854205125418,train perplexity:1.8725205678463968\n",
      "val loss:0.9599593877792358, val perplexity:2.6115904087831447\n",
      "epoch:33,time-mins:0,time-secs:47\n",
      "train loss:0.6023483650806623,train perplexity:1.8264028288167415\n",
      "val loss:0.9306513667106628, val perplexity:2.536160610309776\n",
      "epoch:34,time-mins:0,time-secs:47\n",
      "train loss:0.583002310914871,train perplexity:1.7914087310467597\n",
      "val loss:0.9236904382705688, val perplexity:2.5185678796954862\n",
      "epoch:35,time-mins:0,time-secs:47\n",
      "train loss:0.5596970973106531,train perplexity:1.7501422971916227\n",
      "val loss:0.8875752091407776, val perplexity:2.429232124194898\n",
      "epoch:36,time-mins:0,time-secs:47\n",
      "train loss:0.5444336709303733,train perplexity:1.7236319630292012\n",
      "val loss:0.8626664280891418, val perplexity:2.36947030007612\n",
      "epoch:37,time-mins:0,time-secs:47\n",
      "train loss:0.5348413089911143,train perplexity:1.7071773070680856\n",
      "val loss:0.8390709161758423, val perplexity:2.3142158771228347\n",
      "epoch:38,time-mins:0,time-secs:47\n",
      "train loss:0.5133049346697636,train perplexity:1.6708039782277895\n",
      "val loss:0.8500564098358154, val perplexity:2.3397788347433037\n",
      "epoch:39,time-mins:0,time-secs:47\n",
      "train loss:0.5012474144116427,train perplexity:1.6507791926478779\n",
      "val loss:0.8482232689857483, val perplexity:2.335493619482791\n",
      "epoch:40,time-mins:0,time-secs:47\n",
      "train loss:0.4855752644630579,train perplexity:1.6251096078036147\n",
      "val loss:0.8086514472961426, val perplexity:2.244878607380937\n",
      "epoch:41,time-mins:0,time-secs:47\n",
      "train loss:0.4689601231843997,train perplexity:1.598331261122854\n",
      "val loss:0.8095425367355347, val perplexity:2.246879886527623\n",
      "epoch:42,time-mins:0,time-secs:47\n",
      "train loss:0.4513064389809584,train perplexity:1.570362428630775\n",
      "val loss:0.767194390296936, val perplexity:2.153715284924575\n",
      "epoch:43,time-mins:0,time-secs:47\n",
      "train loss:0.44682120990294677,train perplexity:1.5633347655237737\n",
      "val loss:0.7783792614936829, val perplexity:2.177939532988164\n",
      "epoch:44,time-mins:0,time-secs:47\n",
      "train loss:0.4303723822037379,train perplexity:1.537830077490446\n",
      "val loss:0.7978817820549011, val perplexity:2.220831737067859\n",
      "epoch:45,time-mins:0,time-secs:47\n",
      "train loss:0.4208810058159706,train perplexity:1.5233030034257113\n",
      "val loss:0.7420151233673096, val perplexity:2.1001633414918146\n",
      "epoch:46,time-mins:0,time-secs:47\n",
      "train loss:0.41320131508967817,train perplexity:1.5116493131211979\n",
      "val loss:0.7352124452590942, val perplexity:2.0859250903326116\n",
      "epoch:47,time-mins:0,time-secs:47\n",
      "train loss:0.3954751021586932,train perplexity:1.4850895926100145\n",
      "val loss:0.726765513420105, val perplexity:2.0683796302492574\n",
      "epoch:48,time-mins:0,time-secs:47\n",
      "train loss:0.38467052005804503,train perplexity:1.4691301927595444\n",
      "val loss:0.7129178047180176, val perplexity:2.0399347146429347\n",
      "epoch:49,time-mins:0,time-secs:47\n",
      "train loss:0.3813452355754681,train perplexity:1.4642530303969454\n",
      "val loss:0.6975297927856445, val perplexity:2.0087844598314573\n",
      "epoch:50,time-mins:0,time-secs:47\n",
      "train loss:0.3678928474203134,train perplexity:1.4446872287164707\n",
      "val loss:0.6953747868537903, val perplexity:2.0044601785041745\n"
     ]
    }
   ],
   "source": [
    "from torch.utils.tensorboard import SummaryWriter\n",
    "writer = SummaryWriter(os.getcwd()+'/log', comment='intent_slot')\n",
    "\n",
    "\n",
    "encoder_embedding_dim = 128\n",
    "decoder_embedding_dim = 128\n",
    "hidden_dim = 256\n",
    "n_layers = 1\n",
    "encoder_dropout = 0.5\n",
    "decoder_dropout = 0.5\n",
    "lr = 0.1\n",
    "gamma = 0.1\n",
    "weight_decay = 0.1\n",
    "n_epochs = 50\n",
    "clip = 5.0\n",
    "model_path = os.path.join(os.getcwd(), \"model.h5\")\n",
    "\n",
    "model, optimizer, scheduler, loss_slot, loss_intent = build_model(SOURCE,\n",
    "                                                  TARGET,\n",
    "                                                  LABEL,\n",
    "                                                  encoder_embedding_dim,\n",
    "                                                  decoder_embedding_dim,\n",
    "                                                  hidden_dim,\n",
    "                                                  n_layers,\n",
    "                                                  encoder_dropout,\n",
    "                                                  decoder_dropout,\n",
    "                                                  lr,\n",
    "                                                  gamma,\n",
    "                                                  weight_decay)\n",
    "\n",
    "model, optimizer = amp.initialize(model, optimizer, opt_level='O1')\n",
    "\n",
    "train_model(model,\n",
    "            train_iter,\n",
    "            val_iter,\n",
    "            optimizer,\n",
    "            scheduler,\n",
    "            loss_slot, \n",
    "            loss_intent,\n",
    "            n_epochs,\n",
    "            clip,\n",
    "            model_path,\n",
    "            writer)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}